#!/usr/bin/env python
#coding=utf-8

import requests
from lxml import html as H
from dateutil.parser import parse
import sys

def debug(x):
    sys.stderr.write(x)
    sys.stderr.write("\n")


def get_commit_time(dom):
    commit_time = dom.xpath("//tr[3]//td[2]")[0].text_content()
    time = parse(commit_time)
    return time



def last_commit_time(libname):
    url = "https://android.googlesource.com/platform/external/{}/+/master".format(libname)

    text = requests.get(url).text
    dom = H.fromstring(text)

    return get_commit_time(dom)
    # Thu Jun 29 01:08:57 2017 +0000


def get_diff_files(dom):
    items = dom.xpath("//ul[@class='DiffTree']/li")
    diff_files = []
    # print items
    for item in items:
        a_ele = item.getchildren()[0]
        diff_files.append(a_ele.text_content())
    return diff_files

def last_code_commit_time(libname):
    suffix = [".c", ".java", ".cpp", ".asm", ".pl", ".py", ".rb", ".scala", ".cs", ".go"]
    lib_page_url = "https://android.googlesource.com/platform/external/{}/".format(libname)
    text = requests.get(lib_page_url).text
    dom = H.fromstring(text)

    lis = dom.xpath("//li[@class='CommitLog-item CommitLog-item--oneline']")
    # print lis
    hrefs = []
    for li in lis:
        hrefs.append(li.getchildren()[0].attrib["href"])

    for href in hrefs:
        url = "https://android.googlesource.com{}".format(href)
        # print url
        text = requests.get(url).text
        dom = H.fromstring(text)
        diff_files = get_diff_files(dom)
        # print diff_files
        if any([x for x in diff_files if any([t for t in suffix if x.endswith(t)])]):
            return get_commit_time(dom)


def CveSearch(libname):
    url = "https://cve.mitre.org/cgi-bin/cvekey.cgi?keyword={}".format(libname)
    text = requests.get(url).text
    dom = H.fromstring(text) 
    trs = dom.xpath("//div[@id='CenterPane']//table/tr")
    CVEs = []
    for tr in trs:
        tds= tr.xpath("td[1]")[0]
        if len(tds):
            cve = tds[0].text_content().strip()
            if cve:
                CVEs.append(cve)
    # print CVEs
    return CVEs

def CveReportTime(CveId):
    url = "https://cve.mitre.org/cgi-bin/cvename.cgi?name={}".format(CveId)
    text = requests.get(url).text
    dom = H.fromstring(text)
    trs = dom.xpath("//div[@id='GeneratedTable']/table//tr[9]")[0]
    createDate = trs.xpath("td[1]")[0].text_content()
    return  parse(createDate)



def compareAllCommitTimeAndCveReportTime():
    """
    For all external libraries in AOSP, check their latest commit on master branch that contains code files change
    Compare this commit time with each related CVE report time in CVEdb 
    Warn on each CVE that report time later than latest commit time
    """
    # libs = ["aac", "android-clat", "android-mock", "androidplot", "ant-glob", "antlr", "apache-commons-math", "apache-harmony", "apache-http", "apache-xml", "archive-patcher", "autotest", "avahi", "bison", "blktrace", "boringssl", "bouncycastle", "bsdiff", "bzip2", "caliper",
    # "c-ares", "cblas", "ceres-solver", "chromium-libpac", "chromium-trace", "chromium-webview", "clang", "cmockery", "compiler-rt", "conscrypt", "crcalc", "cros", "curl", "dagger2", "dbus", "dbus-binding-generator", "deqp", "dexmaker", "dhcpcd-6.8.2", "dlmalloc", "dng_sdk",
    # "dnsmasq", "doclava", "donuts", "drm_gralloc", "drm_hwcomposer", "droiddriver", "e2fsprogs", "easymock", "eclipse-basebuilder", "eclipse-windowbuilder", "eigen", "elfutils", "embunit", "emma", "esd", "expat", "eyes-free", "f2fs-tools", "fdlibm", "fec", "fio", "flac",
    # "fonttools", "freetype", "fsck_msdos", "gemmlowp", "giflib", "glide", "gmock", "google-benchmark", "google-breakpad", "google-fonts", "google-tv-pairing-protocol", "gptfdisk", "gtest", "guava", "guice", "hamcrest", "harfbuzz_ng", "hyphenation-patterns", "icu",
    # "ImageMagick", "ims", "iproute2", "ipsec-tools", "iptables", "iputils", "iw", "jacoco", "jarjar", "javasqlite", "javassist", "jcommander", "jdiff", "jemalloc", "jetty", "jhead", "jmdns", "jmonkeyengine", "jsilver", "jsmn", "jsoncpp", "jsr305", "jsr330", "junit",
    # "kernel-headers", "ksoap2", "libavc", "libbrillo", "libcap", "libcap-ng", "libchrome", "libcxx", "libcxxabi", "libdaemon", "libdivsufsort", "libdrm", "libedit", "libevent", "libexif", "libgdx", "libgsm", "libhevc", "libjpeg-turbo", "liblzf", "libmicrohttpd", "libmpeg2",
    # "libmtp", "libnfc-nci", "libnfc-nxp", "libnl", "libogg", "libopus", "libpcap", "libphonenumber", "libpng", "libselinux", "libunwind", "libunwind_llvm", "libusb", "libusb-compat", "libutf", "libvncserver", "libvorbis", "libvpx", "libvterm", "libweave", "libxml2",
    # "libyuv", "littlemock", "lld", "lldb", "llvm", "ltrace", "lz4", "lzma", "marisa-trie", "markdown", "mdnsresponder", "mesa3d", "messageformat", "minijail", "mksh", "mmc-utils", "mockftpserver", "mockito", "mockwebserver", "modp_b64", "mp4parser", "mtpd", "nanohttpd",
    # "nanopb-c", "naver-fonts", "netcat", "netperf", "neven", "nfacct", "nist-pkits", "nist-sip", "noto-fonts", "oauth", "objenesis", "okhttp", "opencv", "opencv3", "openfst", "openssh", "owasp", "parameter-framework", "pcre", "pdfium", "piex", "ppp", "proguard", "protobuf",
    # "regex-re2", "replicaisland", "rmi4utils", "robolectric", "roboto-fonts", "rootdev", "safe-iop", "scrypt", "selinux", "sfntly", "shflags", "skia", "sl4a", "slf4j", "smali", "snakeyaml", "sonic", "sonivox", "speex", "sqlite", "squashfs-tools", "srtp", "strace", "svox",
    # "tagsoup", "tcpdump", "testng", "timezonepicker-support", "tinyalsa", "tinycompress", "tinyxml", "tinyxml2", "tlsdate", "toybox", "tpm2", "tremolo", "unicode", "universal-tween-engine", "v8", "valgrind", "vboot_reference", "vixl", "vogar", "vulkan-validation-layers",
    # "webp", "webrtc", "wpa_supplicant_8", "xmlrpcpp", "xmlwriter", "xmp_toolkit", "zlib", "zopfli", "zxing"
    # ]
    libs = [
       "xmlwriter"
    ]
    for libname in libs:
        debug(libname)
        # libname = "libxml2"
        try:
            lct = last_code_commit_time(libname)
            if not lct:
                sys.stdout.write("library {} not confirmed".format(libname))
                continue
            for cve in CveSearch(libname):
                lct = lct.replace(tzinfo=None)
                crt = CveReportTime(cve)
                # print lct, crt
                if crt > lct:
                    sys.stdout.write("Warn in {}: {} reported at {} after last commit {}\n".format(libname, cve, crt, lct))
        except Exception,e:
            print e
            import traceback
            traceback.print_exc()
            continue

        # break


def commitTimeTest():
    print  last_code_commit_time("pcre")

commitTimeTest()